#!/usr/bin/python
# -*- coding: utf-8 -*-

import glob, sys, os, re, string
import pepe.p_txt.bigtxt as bt
import pepe.p_string.utf8ascii as utf8
import pepe.p_string.special_char_substitutes as schs
import pepe.glib as g
from pepe.utils import u

def headerLine(in_s, enc):
    ''' Converts given string <in_s> into normalized-header form.
    e.g.  'column1    second COLUMN     ěščřžýáíé'
          'COLUMN1_   SECOND_COLUMN_    ESCRZYAIE_'
    '''
    delimiter = g.findcoldelimiter_line(in_s)
    items = in_s.split(delimiter)
    out_s = ''

    for item in items: 
        if enc.lower() == 'utf8' or enc.lower() == 'utf-8':
            item = utf8.uppercase(item)
        else:
            item = schs.plain(item, enc).upper()
        # fastech max column lenght = 32
        out_s += re.sub('_{1,}','_', (re.sub('\W{1,}', '_', item[:30]) + '_\t'))

    return out_s + '\n'
                
def unite(ext, sourceEnc='', start ='zza'):
    ''' Merging of text files depending on files header structure.
    '
    ' Merges files with given mask into files of following pattern 'unite-headerX.dat' 
    ' depending on files header structure. X in the output file pattern represents integer. 
    '
    ' If <sourceEnc> is set up to input file encoding, procedure headerLine() will be called for fist row/header.
    '
    '   @param <ext> string e.g. all *.<ext> will be proceed
    '   @param <sourceEnc> string e.g. headerLine() will be called and the line will be
    '          normalized using p_string.<sourceEnc>ascii.py e.g. 'cp1250', 'utf-8', 'cp852'
    '   @param <start> defines starting position for abc output ranaming.
    '''
    print 'Divinding Files Into Header Groups...\n'
    header_list = open('#header_list.csv', 'w')
    header_list.write("source_file\theader_type\n")
    headers = {}
    fno = 0 # file number counter

    for inpfile in glob.glob(ext): # processing every <ext> files
        lino = 0 # line number counter initialisation
        for line in bt.lines(inpfile): # processing every line within the file
            
            if lino == 0: # header is stripped and used as key in <headers>
                if sourceEnc != '':
                    line = headerLine(line, sourceEnc)
                if 1:
                    if re.match('\w{3}_\d{3}_\d{3}_\d{3}', line[:15]): # check whether file contains unique ID
                        line = 'UNIQUE_ID' + line.strip()[16:] + "\n"                  
                    else:
                        #print inpfile + '-> UNIQUE ID NOT FOUND'
                        line = line.strip() + "\n"
                if headers.has_key(line):
                     of = open(headers[line], 'a')                    
                else: # creating new output file otherwise
                    header_group = str(len(headers)+1)
                   
                    headers[line] = "unite-header-" + header_group + ".dat"
                    of = open(headers[line], 'w')
                    of.write('HEADER_GROUP\t' + line)

                # writing list of files with group number
                header_list.write(inpfile + '\t' + re.sub('\D','',headers[line])+'\n') 

            else: # every non-header line is writen into output

                line = re.sub('\n','',line)
                if lino == 1:                     
                    of_group = re.sub('\D','',str(of)[25:27])
                    of.write(of_group + '\t' + line)
                else:
                    of.write('\n'+ of_group +'\t' + line)   

            lino += 1
        print inpfile + '......Done'
        of.write('\n') # otherwise row from nech wile will not start on new line
        of.close() # after processing every rows, output is closed and
        fno += 1 # opened (w or a) again with a new line of next file

    for hdr in headers:
        header_list.write('\t'+ re.sub('\D','',headers[hdr]) + '\t' + hdr)      # writing header list
    print '\n'+ str(fno) + ' Files Processed \n\n Renaming...'

    u('data-rename-abc', 'unite-header*.dat', start)
    
def main():
    pass

if __name__ == "__main__":
    """ """
    if len(sys.argv) == 2:
        unite(ext=sys.argv[1], sourceEnc='utf-8', start ='zza')
    elif len(sys.argv) == 3:
        unite(ext=sys.argv[1], sourceEnc=sys.argv[2], start ='zza')
    elif len(sys.argv) == 4:
        unite(ext=sys.argv[1], sourceEnc=sys.argv[2], start=sys.argv[3])
    else:
        print """

    Merging of text files depending on files header structure.

    unite.py <files-mask> [<source-enc>] [<start-output-name>]

    example : unite *.dat
    example : unite *.dat utf-8
    example : unite *.dat cp852 zza
    example : unite *.dat cp1250 zza
    example : unite *.dat cp1252 zza
    example : unite *.dat 8859-1 zza
    example : unite *.dat 8859-2 zza
    example : unite *.dat Mac-CE zza

    <files-mask> (mandatory)
        files to be processed (merged)

    <source-enc> (optional)
        defines source files encoding (to properly recode headers lines to ASCII)
        default 'utf-8' or
                'cp1250', 'utf-8', 'cp852' ...

    <start-output-name> (optional)
        defines starting position for abc style of output renaming
        default 'zza' (firt output file will be named zza, second zzb ...)

    """
